import db from './db/inedx.js'
import { crawlWeb } from '../../utils/crawler.js'
import { saveJson, sleep, random, readJson } from '../../utils/util.js'
import { isDev, dataDir, techniqueDir, subTechniqueDir } from './config/index.js'

import Page from './page.js'

const techniques = await readJson(`${dataDir}/technique.json`)

class SubTechnique {
  constructor() {
    this.page = new Page()
  }

  get db() {
    return db.projectConfig
  }

  get config() {
    return db.projectConfig.data
  }

  // 开始任务
  async start() {
    await this.crawList()
  }

  // 抓取列表
  async crawList() {
    const subTechniqueTotal = techniques.length
    const { subTechniqueIndex, subTechniqueContentIndex } = this.config

    console.log(
      `当前进度:${subTechniqueIndex}/${subTechniqueTotal} ${(subTechniqueIndex / subTechniqueTotal) * 100}% `
    )

    // 抓取子攻击
    const subTechniques = await this.getsubTechniques()
    for (let i = subTechniqueContentIndex; i < subTechniques.length; i++) {
      console.log(`子页面${i}/${subTechniques.length}`, subTechniques[i].url)

      const subTechnique = subTechniques[i]
      await this.crawlContent(subTechnique.url, subTechnique)
    }

    this.config.subTechniqueIndex += 1
    this.config.subTechniqueContentIndex = 0
    await this.db.write()

    if (subTechniqueIndex < subTechniqueTotal) await this.crawList()
  }

  // 抓取内容
  async crawlContent(url, item) {
    await sleep(random(5000, 10000)) // 延迟1-5秒执行
    const { $ } = await crawlWeb(url)
    const json = this.page.getContentJson($)
    const data = Object.assign(item, json)

    delete data.subTechniques // 子攻击过滤掉子攻击

    saveJson(`${subTechniqueDir}/${item.id}.json`, data)

    this.config.subTechniqueContentIndex += 1
    await this.db.write()
  }

  async getsubTechniques() {
    const index = this.config.subTechniqueIndex
    const item = techniques[index]
    const techniqueData = await readJson(`${techniqueDir}/${item.id}.json`)

    return techniqueData.subTechniques
  }
}

export default SubTechnique
